{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "sx9e_pXlCuti"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "from torch.optim import lr_scheduler\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "from torchvision.datasets import MNIST\n",
    "from torchvision import transforms\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UMMut8UVCutt"
   },
   "source": [
    "# 1. Setup and initializations\n",
    "We'll go through analysing the role of Dropout on MNIST dataset. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExperimentParams():\n",
    "    def __init__(self):\n",
    "        self.data_dir = '/home/docker_user/'\n",
    "        self.num_classes = 10\n",
    "        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        self.batch_size = 256\n",
    "        self.num_epochs = 20\n",
    "        self.num_workers = 4\n",
    "        self.lr = 1e-2\n",
    "        \n",
    "        self.drop_prob1 = 0.2\n",
    "        self.drop_prob2 = 0.5\n",
    "\n",
    "args = ExperimentParams()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BcmGBqXeCutw"
   },
   "source": [
    "## 1.1 Prepare dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "AQSHPH9P0BNx"
   },
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[Errno 45] Operation not supported: '/home/docker_user'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-04471a49ecea>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m                              transform=transforms.Compose([\n\u001b[1;32m      5\u001b[0m                                  \u001b[0mtransforms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mToTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                                  \u001b[0mtransforms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m                              ]))\n\u001b[1;32m      8\u001b[0m test_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=False, download=True,\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, root, train, transform, target_transform, download)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdownload\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_exists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;31m# download files\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocessed_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m         \u001b[0mmkdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0;31m# Cannot rely on checking for EEXIST, since the operating system\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [Errno 45] Operation not supported: '/home/docker_user'"
     ]
    }
   ],
   "source": [
    "mean, std = 0.1307, 0.3081\n",
    "\n",
    "train_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=True, download=True,\n",
    "                             transform=transforms.Compose([\n",
    "                                 transforms.ToTensor(),\n",
    "                                 transforms.Normalize((mean,), (std,))\n",
    "                             ]))\n",
    "test_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=False, download=True,\n",
    "                            transform=transforms.Compose([\n",
    "                                transforms.ToTensor(),\n",
    "                                transforms.Normalize((mean,), (std,))\n",
    "                            ]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TcZTFRnjCut3"
   },
   "source": [
    "## 1.2 Common setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Dz2xh66UCut5"
   },
   "outputs": [],
   "source": [
    "\n",
    "mnist_classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
    "\n",
    "\n",
    "def get_raw_images(dataloader,mean=0.1307, std=0.3081):\n",
    "\n",
    "    raw_images = np.zeros((len(dataloader.dataset), 1, 28, 28))\n",
    "    k = 0\n",
    "    for input, target in dataloader:\n",
    "        raw_images[k:k+len(input)] = (input*std + mean).data.cpu().numpy()\n",
    "        k += len(input)\n",
    "\n",
    "    return raw_images\n",
    "\n",
    "\n",
    "def show(img, title=None):\n",
    "    # img is a torch.Tensor     \n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')\n",
    "    plt.axis('off')\n",
    "    if title is not None:\n",
    "        plt.title(title)\n",
    "    plt.pause(0.001)  # pause a bit so that plots are updated"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "75moY8AyCut_"
   },
   "source": [
    "# 2. Playing with DropOut\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise\n",
    "\n",
    "Complete the missing blocks in the definition of the following `DropoutNet` architecture: (`FullyConnected 256 -> ReLU -> Dropout (0.2) -> Fully Connected 256 -> ReLU -> -> Dropout (0.5) -> Fully Connected 10 `)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DropoutNet(nn.Module):\n",
    "    def __init__(self, num_classes=10,drop_prob1=0.2, drop_prob2=0.5):\n",
    "        super(DropoutNet, self).__init__()\n",
    "        self.classifier = nn.Sequential( \n",
    "                    #  TODO\n",
    "                    )\n",
    "\n",
    "    def forward(self, x):\n",
    "                # TODO\n",
    "        return self.classifier(x)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up data loaders\n",
    "\n",
    "kwargs = {'num_workers': args.num_workers, 'pin_memory': True} \n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, **kwargs)\n",
    "\n",
    "model_dropout = DropoutNet(num_classes=args.num_classes, drop_prob1=args.drop_prob1, drop_prob2=args.drop_prob2)\n",
    "optimizer_dropout = optim.Adam(model_dropout.parameters(), lr=args.lr)\n",
    "scheduler_dropout = lr_scheduler.StepLR(optimizer_dropout, 8, gamma=0.1, last_epoch=-1)\n",
    "\n",
    "model_simple = DropoutNet(num_classes=args.num_classes, drop_prob1=0, drop_prob2=0)\n",
    "optimizer_simple = optim.Adam(model_simple.parameters(), lr=args.lr)\n",
    "scheduler_simple = lr_scheduler.StepLR(optimizer_dropout, 8, gamma=0.1, last_epoch=-1)\n",
    "\n",
    "loss_fn = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "model_dropout.to(args.device)\n",
    "model_simple.to(args.device)\n",
    "loss_fn.to(args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_classif_epoch(train_loader, model, loss_fn, optimizer, args, log_interval=100):\n",
    "    model.train()\n",
    "    losses = []\n",
    "    total_loss, total_corrects, num_samples = 0, 0, 0\n",
    "    corrects = 0.    \n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        num_samples += data.size(0)\n",
    "        \n",
    "        data, target = data.to(args.device), target.to(args.device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(data)\n",
    "\n",
    "        loss = loss_fn(outputs, target)\n",
    "        losses.append(loss.data.item())\n",
    "\n",
    "        _,preds = torch.max(outputs.data,1)\n",
    "        corrects += torch.sum(preds == target.data).cpu()\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if batch_idx % log_interval == 0:\n",
    "            print('Train: [{}/{} ({:.0f}%)]\\tLoss: {:.6f} \\tAccuracy: {}'.format(\n",
    "                batch_idx * len(data[0]), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), np.mean(losses), float(total_corrects)/num_samples))           \n",
    "            \n",
    "            total_loss += np.sum(losses)\n",
    "            total_corrects += corrects\n",
    "            losses, corrects = [], 0\n",
    "            \n",
    "    accuracy = total_corrects.item()/num_samples\n",
    "    return total_loss/(batch_idx + 1), accuracy\n",
    "\n",
    "def test_classif_epoch(test_loader, model, loss_fn, args, log_interval=100):\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        losses, corrects = [], 0\n",
    "        num_samples = 0\n",
    "        corrects = 0.\n",
    "        for batch_idx, (data, target) in enumerate(test_loader):\n",
    "\n",
    "            num_samples += data.size(0)\n",
    "            data, target = data.to(args.device), target.to(args.device)\n",
    "\n",
    "            outputs = model(data)\n",
    "\n",
    "            loss = loss_fn(outputs, target)\n",
    "            losses.append(loss.data.item())\n",
    "\n",
    "            _,preds = torch.max(outputs.data,1)\n",
    "            corrects += torch.sum(preds == target.data).cpu()\n",
    "\n",
    "        accuracy = corrects.item()/num_samples\n",
    "        return np.sum(losses)/(batch_idx + 1), accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training the baseline model for a while"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "\n",
    "for epoch in range(0, start_epoch):\n",
    "    scheduler_simple.step()\n",
    "\n",
    "train_losses_simple, val_losses_simple, val_accuracies_simple = [], [], []\n",
    "for epoch in range(start_epoch, args.num_epochs):\n",
    "    scheduler_simple.step()\n",
    "\n",
    "    train_loss, train_accuracy = train_classif_epoch(train_loader, model_simple, loss_fn, optimizer_simple, args)\n",
    "\n",
    "    message = 'Epoch: {}/{}. Train set: Average loss: {:.4f} Average accuracy: {:.4f}'.format(\n",
    "        epoch + 1, args.num_epochs, train_loss, train_accuracy)\n",
    "    \n",
    "    val_loss, val_accuracy = test_classif_epoch(test_loader, model_simple, loss_fn, args)\n",
    "    \n",
    "    message += '\\nEpoch: {}/{}. Validation set: Average loss: {:.4f}  Average accuracy: {:.4f}'.format(epoch + 1, args.num_epochs,\n",
    "                                                                             val_loss, val_accuracy)\n",
    "    print(message)\n",
    "    train_losses_simple.append(train_loss)\n",
    "    val_losses_simple.append(val_loss)\n",
    "    val_accuracies_simple.append(val_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training the Dropout variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "\n",
    "for epoch in range(0, start_epoch):\n",
    "    scheduler_dropout.step()\n",
    "\n",
    "train_losses, val_losses, val_accuracies = [], [], []\n",
    "for epoch in range(start_epoch, args.num_epochs):\n",
    "    scheduler_dropout.step()\n",
    "\n",
    "    train_loss, train_accuracy = train_classif_epoch(train_loader, model_dropout, loss_fn, optimizer_dropout, args)\n",
    "\n",
    "    message = 'Epoch: {}/{}. Train set: Average loss: {:.4f} Average accuracy: {:.4f}'.format(\n",
    "        epoch + 1, args.num_epochs, train_loss, train_accuracy)\n",
    "    \n",
    "    val_loss, val_accuracy = test_classif_epoch(test_loader, model_dropout, loss_fn, args)\n",
    "    \n",
    "    message += '\\nEpoch: {}/{}. Validation set: Average loss: {:.4f}  Average accuracy: {:.4f}'.format(epoch + 1, args.num_epochs,\n",
    "                                                                             val_loss, val_accuracy)\n",
    "    print(message)\n",
    "    train_losses.append(train_loss)\n",
    "    val_losses.append(val_loss)\n",
    "    val_accuracies.append(val_accuracy)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#### Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.cla()\n",
    "epochs = np.arange(args.num_epochs)\n",
    "plt.plot(epochs, train_losses_simple, 'orange', lw=3, label='no dropout')\n",
    "plt.plot(epochs, train_losses, 'green', lw=3, label='with dropout')\n",
    "plt.legend(loc='upper left'); \n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('train loss')\n",
    "plt.title('Train loss')\n",
    "plt.grid(True)\n",
    "plt.pause(0.1)\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.cla()\n",
    "epochs = np.arange(args.num_epochs)\n",
    "plt.plot(epochs, val_losses_simple, 'orange', lw=3, label='no dropout')\n",
    "plt.plot(epochs, val_losses, 'green', lw=3, label='with dropout')\n",
    "plt.legend(loc='upper left'); \n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('validation loss')\n",
    "plt.title('Validation loss')\n",
    "plt.grid(True)\n",
    "plt.pause(0.1)\n",
    "plt.show()\n",
    "\n",
    "        \n",
    "plt.cla()\n",
    "epochs = np.arange(args.num_epochs)\n",
    "plt.plot(epochs, val_accuracies_simple, 'orange', lw=3, label='no dropout')\n",
    "plt.plot(epochs, val_accuracies, 'green', lw=3, label='with dropout')\n",
    "plt.legend(loc='upper left'); \n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('validation accuracy')\n",
    "plt.title('Validation accuracy')\n",
    "plt.grid(True)\n",
    "plt.pause(0.1)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Try out what happens if you change the dropout probabilities for layers 1 and 2. In particular, what happens if you switch the ones for both layers?\n",
    "2. Increase the number of epochs and compare the results obtained when using dropout with those when not using it.\n",
    "3. If changes are made to the model to make it more complex, such as adding hidden layer units, will the effect of using dropout to cope with overfitting be more obvious?\n",
    "4. What happens if we apply dropout to the individual weights of the weight matrix rather than the activations?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "Experiments_MNIST.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
